{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]]),\n",
       " array([2, 1, 0, 1, 0]),\n",
       " (800, 784),\n",
       " (800,))"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "#加载数据\n",
    "def load_data():\n",
    "    #这个数据集,只有数字0,1,2,3,一共800行784列\n",
    "    with open('mnist_cut.csv') as fr:\n",
    "        lines = fr.readlines()\n",
    "\n",
    "    x = np.empty((len(lines), 784), dtype=float)\n",
    "    y = np.empty(len(lines), dtype=int)\n",
    "\n",
    "    for i in range(len(lines)):\n",
    "        line = lines[i].strip().split(',')\n",
    "        x[i] = line[1:]\n",
    "        y[i] = line[0]\n",
    "\n",
    "    #归一化\n",
    "    x /= 255\n",
    "\n",
    "    return x, y\n",
    "\n",
    "\n",
    "x, y = load_data()\n",
    "x[:5], y[:5], x.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 168\n",
      "1 225\n",
      "2 209\n",
      "3 198\n"
     ]
    }
   ],
   "source": [
    "#查看各类的数量\n",
    "for i in range(4):\n",
    "    print(i, (y == [i]).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "N, M = x.shape\n",
    "\n",
    "#这里训练的是ovr的多分类,一共4个二分类器\n",
    "params = [0, 1, 2, 3]\n",
    "ws = {}\n",
    "bs = {}\n",
    "for i in params:\n",
    "    ws[i] = np.empty(M)\n",
    "    ws[i].fill(1 / M)\n",
    "    bs[i] = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5360144532148693"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#预测函数,显然,他一次只能做一次二分类判断\n",
    "def predict(params, x):\n",
    "    #print('predict,params=', params)\n",
    "    z = ws[params].dot(x) + bs[params]\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "\n",
    "predict(2, x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-537.0785022432013"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#这个log函数是为了避免log0.\n",
    "def log(p):\n",
    "    if p < 1e-20:\n",
    "        p = 1e-20\n",
    "    return np.log(p)\n",
    "\n",
    "\n",
    "#loss函数\n",
    "def get_loss(param):\n",
    "    loss = 0\n",
    "    for i in range(N):\n",
    "        p = predict(param, x[i])\n",
    "        #等于param的类是0，否则类是1\n",
    "        d = 0 if y[i] == param else 1\n",
    "        loss += d * log(p) + (1 - d) * log(1 - p)\n",
    "\n",
    "    return loss\n",
    "\n",
    "\n",
    "get_loss(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00, -1.06164180e-02,\n",
       "       -1.67044141e-01, -7.56527356e-01, -1.37879160e+00, -1.28721198e+00,\n",
       "       -6.87102499e-01, -9.48565992e-01, -1.90487426e+00, -1.90946797e+00,\n",
       "       -2.06211729e+00, -7.12597409e-01, -1.18957178e-01, -1.48696358e-02,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        0.00000000e+00, -1.06773007e-01, -6.04983927e-01, -1.93870553e+00,\n",
       "       -3.78568471e+00, -6.35685791e+00, -1.01082533e+01, -1.18149299e+01,\n",
       "       -1.22961081e+01, -1.50634850e+01, -1.68083432e+01, -1.42151627e+01])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#暴力求梯度法\n",
    "#同样的,因为这个程序中实际是4个二分类器,所以这里的梯度,也要指定是哪一个的\n",
    "def get_gradient(param):\n",
    "    global w\n",
    "    global b\n",
    "    upsilon = 1e-5\n",
    "\n",
    "    gradient_w = np.empty(M)\n",
    "\n",
    "    for i in range(M):\n",
    "        l1 = get_loss(param)\n",
    "        ws[param][i] += upsilon\n",
    "        l2 = get_loss(param)\n",
    "        ws[param][i] -= upsilon\n",
    "        gradient_w[i] = (l2 - l1) / upsilon\n",
    "\n",
    "    l1 = get_loss(param)\n",
    "    bs[param] += upsilon\n",
    "    l2 = get_loss(param)\n",
    "    bs[param] -= upsilon\n",
    "    gradient_b = (l2 - l1) / upsilon\n",
    "\n",
    "    return gradient_w, gradient_b\n",
    "\n",
    "\n",
    "get_gradient(2)[0][:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0 -632.6963241739369\n",
      "0 10 -38.981318928687415\n",
      "0 20 -28.588793998481414\n",
      "0 30 -23.014555327394415\n",
      "0 40 -19.404857433677375\n",
      "1 0 -580.948153246214\n",
      "1 10 -43.96444525860368\n",
      "1 20 -33.85263049684668\n",
      "1 30 -28.010717257119936\n",
      "1 40 -24.101525607073505\n",
      "2 0 -813.0594134266456\n",
      "2 10 -116.88068344288085\n",
      "2 20 -101.37078871602604\n",
      "2 30 -91.96638887264375\n",
      "2 40 -85.08306111384184\n",
      "3 0 -849.5871861168231\n",
      "3 10 -82.83738995425851\n",
      "3 20 -68.55090861007622\n",
      "3 30 -60.03544809177709\n",
      "3 40 -54.05138493660565\n"
     ]
    }
   ],
   "source": [
    "#训练,因为是4个分类器,显然要训练4次\n",
    "for param in params:\n",
    "    for i in range(50):\n",
    "        g_w, g_b = get_gradient(param)\n",
    "        ws[param] += g_w * 1e-3\n",
    "        bs[param] += g_b * 1e-3\n",
    "\n",
    "        if i % 10 == 0:\n",
    "            print(param, i, get_loss(param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 2)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#最终结果由4个分类器投票表决，如果出现平局可以考虑每个分类器的置信度\n",
    "def vote(x):\n",
    "    for param in params:\n",
    "        p = predict(param, x)\n",
    "        if p < 0.5:\n",
    "            return param\n",
    "    return 0\n",
    "\n",
    "\n",
    "vote(x[0]), y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.96\n"
     ]
    }
   ],
   "source": [
    "#测试\n",
    "correct = 0\n",
    "for i in range(N):\n",
    "    p = vote(x[i])\n",
    "    if p == y[i]:\n",
    "        correct += 1\n",
    "\n",
    "print(correct / N)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
